import math
from jili.tool.state import config as config0
from jili.core.printlog import print
import datetime
import os
from jili.core import load,save
import pandas as pd
import numpy as np
from copy import deepcopy
from tqdm import tqdm
from jili.calc.graph import graph
from jili.data.db import getdb_client
from jili.data.db import get_calender
from jili.tool.convert import str2datetime
from research.factor import factordata
from research.factor import end_date_calcor,gen_tradedata_a,convert_byobj2bydate,fullend_date_calcor,gen_tradedata_a_noobj
from jili.data.db import tradecal as tcl
def gen_group(objs,n):
    rst=[]
    objs.sort()
    for i in range(n):
        rst.append([])
    nn=0
    for obj in objs:
        nnn=nn%n
        rst[nnn].append(obj)
        nn=nn+1
    return rst
def save_new_factordata(name,data,factor_url,freq="%Y%m"):
    if data:
        # url = os.path.join(factor_url, name+".pkl")
        # if os.path.exists(url):
        #     a = load(url)
        #     a.update(data)
        #     save(a, url)
        data1={}
        for date, v in data.items():
            key = date.strftime(freq)
            if key not in data1.keys():
                data1[key] = {}
            data1[key][date] = v
        for key, v in data1.items():
            save_url = os.path.join(factor_url, "bydate", name, key + ".pkl")
            if os.path.exists(save_url):
                a = load(save_url)
                a.update(v)
                save(a, save_url)
            else:
                save(v, save_url)
def get_fullreportdate(date,period=0):
    date0 = str2datetime(date)
    end_dates = [datetime.datetime(date0.year-1, 12, 31),
                 datetime.datetime(date0.year, 6, 30)]
    n=-1
    n = -1
    for i in end_dates:
        if date0 <= i:
            break
        n = n + 1
    n = n + period
    if n >= 0:
        m = 0
        while n > 1:
            m = m + 1
            n = n - 2
        year = date0.year + m
        end_dates0 = [datetime.datetime(year-1, 12, 31),
                 datetime.datetime(year, 6, 30)]
        return end_dates0[n]
    else:
        m = 0
        while n < 0:
            m = m + 1
            n = n + 2
        year = date0.year - m
        end_dates0 = [datetime.datetime(year-1, 12, 31),
                 datetime.datetime(year, 6, 30)]
        return end_dates0[n]
def get_end_date(date,period=0):
    date0=str2datetime(date)
    end_dates=[datetime.datetime(date0.year,3,31),
               datetime.datetime(date0.year,6,30),
               datetime.datetime(date0.year,9,30),
               datetime.datetime(date0.year,12,31)]
    n=-1
    for i in end_dates:
        if date0<=i:
            break
        n=n+1
    n=n+period
    if n>=0:
        m=0
        while n>3:
            m=m+1
            n=n-4
        year=date0.year+m
        end_dates0 = [datetime.datetime(year, 3, 31),
                     datetime.datetime(year, 6, 30),
                     datetime.datetime(year, 9, 30),
                     datetime.datetime(year, 12, 31)]
        return end_dates0[n]
    else:
        m=0
        while n<0:
            m=m+1
            n=n+4
        year=date0.year-m
        end_dates0 = [datetime.datetime(year, 3, 31),
                     datetime.datetime(year, 6, 30),
                     datetime.datetime(year, 9, 30),
                     datetime.datetime(year, 12, 31)]
        return end_dates0[n]
class index_score:
    def update(self,name,factor_url,k1d_url,base_facotrs=["is_wind_new_theme_detail","trade_days","jg_hold_info", "rps_20"],
               start_date=None,arg={"level":87,"freehold_ratio":0.02,"trade_days":180},freq="%Y%m"):
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                self.calc_index_strength(name,factor_url,base_facotrs=base_facotrs,start_date=None,end_date=None,arg=arg, issave=True)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            start_date=maxdate+datetime.timedelta(days=1)
            print("增量计算", maxdate,start_date,maxdate_k)
            data=self.calc_index_strength(name,factor_url,base_facotrs=base_facotrs,start_date=maxdate,end_date=maxdate_k,arg=arg, issave=False)
            save_new_factordata(name,data,factor_url,freq=freq)
    def calc_index_strength(self,name,factor_url,base_facotrs=["is_wind_new_theme_detail","trade_days","jg_hold_info", "rps_20"],
                            start_date=None,end_date=None,arg={"level":87,"freehold_ratio":0.02,"trade_days":180}, issave=False):
        f1 = factordata(base_facotrs[3], url=factor_url)
        f2 = factordata(base_facotrs[1], url=factor_url)
        f3 = factordata(base_facotrs[0], url=factor_url)
        f4 = factordata(base_facotrs[2], url=factor_url)
        if start_date:
            rpsd = {}
            trade_days = {}
            detaild = {}
            top10_holders_jgrate = {}
            for k, v in f1.iter_data(start_date, end_date):
                rpsd[k] = v
            for k, v in f2.iter_data(start_date, end_date):
                trade_days[k] = v
            for k, v in f3.iter_data(start_date, end_date):
                detaild[k] = v
            for k, v in f4.iter_data(start_date, end_date):
                top10_holders_jgrate[k] = v
        else:
            rpsd = f1.get_all()
            trade_days = f2.get_all()
            detaild = f3.get_all()
            top10_holders_jgrate = f4.get_all()
        # rpsd = load(os.path.join(r"G:\factor\k1d", arg[3]))
        # trade_days = load(os.path.join(r"G:\factor\k1d", arg[1]))
        # detaild = load(os.path.join(r"G:\factor\k1d", arg[0]))
        # top10_holders_jgrate = load(os.path.join(r"G:\factor\k1d", arg[2]))
        rst = {}
        for d, v in tqdm(detaild.items()):
            if (d in rpsd.keys()) & (d in trade_days.keys()):
                if d not in rst.keys():
                    rst[d] = {}
                t = rst[d]
                for index, objs in v.items():
                    topn = 0
                    for obj in objs:
                        if obj in rpsd[d].keys():
                            if rpsd[d][obj] >= arg["level"]:
                                if obj not in trade_days[d].keys():
                                    continue
                                if trade_days[d][obj] >= arg["trade_days"]:
                                    try:
                                        if top10_holders_jgrate[d][obj]["freehold_ratio"] >= arg["freehold_ratio"]:
                                            topn = topn + 1
                                    except:
                                        pass
                    t0 = topn ** 2 / len(objs)
                    t[index] = t0
        if issave:
            save_url=os.path.join(factor_url,name)
            save(rst,save_url)
        return rst
class gen_new_wind_index1:
    def update(self,name,factor_url,k1d_url,arg=["is_wind_detail_l1","is_wind_concept_detail_l1"],add_indexs = ['884039.WI','884045.WI'],start_date=None,freq="%Y%m"):
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                self.calc_new_theme_concept_wind_idnex(name,factor_url,arg=arg,add_indexs = add_indexs,start_date=None,end_date=None,issave=True)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            start_date=maxdate+datetime.timedelta(days=1)
            print("增量计算", maxdate,start_date,maxdate_k)
            data=self.calc_new_theme_concept_wind_idnex(name,factor_url,arg=arg,add_indexs = add_indexs,start_date=maxdate,end_date=maxdate_k,issave=False)
            save_new_factordata(name,data,factor_url,freq=freq)
    def calc_new_theme_concept_wind_idnex(self,name,factor_url,arg=["is_wind_detail_l1","is_wind_concept_detail_l1"],add_indexs = ['884039.WI','884045.WI'],start_date=None,end_date=None,issave=False):
        f1 = factordata(arg[0], url=factor_url)
        f2 = factordata(arg[1], url=factor_url)
        if start_date:
            wind_theme_index = {}
            wind_concept_index = {}
            for k, v in f1.iter_data(start_date, end_date):
                wind_theme_index[k] = v
            for k, v in f2.iter_data(start_date, end_date):
                wind_concept_index[k] = v
        else:
            wind_theme_index = f1.get_all()
            wind_concept_index = f2.get_all()
        # wind_theme_index = load(r"G:\factor\k1d\is_wind_detail_l1.pkl")
        # wind_concept_index = load(r"G:\factor\k1d\is_wind_concept_detail_l1.pkl")
        for date in tqdm(wind_theme_index.keys()):
            v = wind_theme_index[date]
            if date in wind_concept_index.keys():
                date_concept = wind_concept_index[date]
                for i in add_indexs:
                    if i in date_concept.keys():
                        wind_theme_index[date][i] = date_concept[i]
                        for stock in date_concept[i]:
                            for theme,v1 in v.items():
                                if (theme != i) & (stock in v1):
                                    # print(date,theme,stock)
                                    v1.remove(stock)
                                    wind_theme_index[date][theme] = v1
        if issave:
            save_url=os.path.join(factor_url,name)
            save(wind_theme_index,save_url)
        return wind_theme_index
class gen_new_wind_index2:
    def update(self,name,factor_url,k1d_url,arg=["is_wind_detail_l1","is_wind_concept_detail_l1"],start_date=None,freq="%Y%m"):
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                self.calc_new_theme_concept_wind_idnex(name,factor_url,arg=arg,start_date=None,end_date=None,issave=True)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            start_date=maxdate+datetime.timedelta(days=1)
            print("增量计算", maxdate,start_date,maxdate_k)
            data=self.calc_new_theme_concept_wind_idnex(name,factor_url,arg=arg,start_date=maxdate,end_date=maxdate_k,issave=False)
            save_new_factordata(name,data,factor_url,freq=freq)
    def calc_new_theme_concept_wind_idnex(self,name,factor_url,arg=["is_wind_detail_l1","is_wind_concept_detail_l1"],start_date=None,end_date=None,issave=False):
        f1 = factordata(arg[0], url=factor_url)
        f2 = factordata(arg[1], url=factor_url)
        if start_date:
            wind_theme_index = {}
            wind_concept_index = {}
            for k, v in f1.iter_data(start_date, end_date):
                wind_theme_index[k] = v
            for k, v in f2.iter_data(start_date, end_date):
                wind_concept_index[k] = v
        else:
            wind_theme_index = f1.get_all()
            wind_concept_index = f2.get_all()

        new_index = {}
        mix_concept_index = pd.read_excel(os.path.join(factor_url,'概念板块.xlsx'),index_col = 0)
        mix_concept_list = list(mix_concept_index.index)

        for date in tqdm(wind_theme_index.keys()):
            v = wind_theme_index[date]
            for concept in mix_concept_list:
                if concept in wind_concept_index[date].keys():
                    if len(wind_concept_index[date][concept]) > 0:
                        v[concept] = wind_concept_index[date][concept]
            if date not in new_index.keys():
                new_index[date] = v
        if issave:
            save_url=os.path.join(factor_url,name)
            save(new_index,save_url)
        return new_index
class is_new_low_inndays:
    def update(self,name,factor_url,k1d_url, arg=["tech_low_min60",'tech_low'], days=5,start_date=None,freq="%Y%m",region="C"):
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                data=self.cal_is_new_low(name,factor_url, arg=arg, days=days,issave=True,start=None,end=None,region=region)
                save_new_factordata(name, data, factor_url, freq=freq)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            #start_date=maxdate+datetime.timedelta(days=1)
            start_date = tcl.get_diff_tradeday(maxdate, -5,region=region)
            print("增量计算", maxdate,start_date,maxdate_k)
            data=self.cal_is_new_low(name,factor_url, arg=arg, days=days,issave=False,start=maxdate,end=maxdate_k,region=region)
            save_new_factordata(name,data,factor_url,freq=freq)
    def cal_is_new_low(self,name,factor_url, arg=["tech_low_min60",'tech_low'], days=5,issave=False,start=None,end=None,region="C"):
        f1=factordata(arg[0],url=factor_url)
        f2 = factordata(arg[1], url=factor_url)
        if start:
            a={}
            a1={}
            start0 = tcl.get_diff_tradeday(start, -days,region=region)
            for k,v in f1.iter_data(start0,end):
                a[k]=v
            for k,v in f2.iter_data(start0,end):
                a1[k] = v
        else:
            a = f1.get_all()
            a1 = f2.get_all()
        rst = {}
        save_url = name
        for d, v in tqdm(a.items()):
            # if (d <= datetime.datetime(2021, 8, 5)) & (d >= datetime.datetime(2000, 1, 1)):
            if (d >= datetime.datetime(2000, 1, 1)):
                f = {}
                for obj, v1 in v.items():
                    if v1:
                        if obj in a1[d].keys():
                            if v1 >= a1[d][obj]:
                                if obj not in f.keys():
                                    f[obj] = {}
                                    f[obj]['is_new_low'] = 1
                rst[d] = f
        rst_fina = {}
        dates = list(rst.keys())
        for d, v in tqdm(rst.items()):
            f = {}
            start1 = dates[(dates.index(d)) - days + 1]
            for d1 in get_calender(start=start1, end=d,region=region):
                for obj, v1 in rst[d1].items():
                    if v1['is_new_low'] >= 1:
                        if obj not in f.keys():
                            f[obj] = 1
                        else:
                            f[obj] = f[obj] + 1
            if start is not None:
                if str2datetime(d)>=str2datetime(start):
                    rst_fina[d] = f
            else:
                rst_fina[d] = f
        # if issave:
        #     save(rst_fina, os.path.join(factor_url, save_url))
        return rst_fina
class is_new_high_inndays:
    def update(self,name,factor_url,k1d_url, arg=["tech_high_max60",'tech_high'], days=5,start_date=None,freq="%Y%m",region="C"):
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                data=self.cal_is_new_high(name,factor_url, arg=arg, days=days,issave=True,start=None,end=None,region=region)
                save_new_factordata(name, data, factor_url, freq=freq)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            #start_date=maxdate+datetime.timedelta(days=1)
            start_date = tcl.get_diff_tradeday(maxdate, -5,region=region)
            print("增量计算", maxdate,start_date,maxdate_k)
            data=self.cal_is_new_high(name,factor_url, arg=arg, days=days,issave=False,start=maxdate,end=maxdate_k,region=region)
            save_new_factordata(name,data,factor_url,freq=freq)
    def cal_is_new_high(self,name,factor_url, arg=["tech_high_max60",'tech_high'], days=5,issave=False,start=None,end=None,region="C"):

        f1=factordata(arg[0],url=factor_url)
        f2 = factordata(arg[1], url=factor_url)
        if start:
            a={}
            a1={}
            start0 = tcl.get_diff_tradeday(start, -days,region=region)
            for k,v in f1.iter_data(start0,end):
                a[k]=v
            for k,v in f2.iter_data(start0,end):
                a1[k] = v
        else:
            a = f1.get_all()
            a1 = f2.get_all()
        rst = {}
        save_url = name
        for d, v in tqdm(a.items()):
            # if (d <= datetime.datetime(2021, 8, 5)) & (d >= datetime.datetime(2000, 1, 1)):
            if (d >= datetime.datetime(2000, 1, 1)):
                f = {}
                for obj, v1 in v.items():
                    if v1:
                        if obj in a1[d].keys():
                            if v1 <= a1[d][obj]:
                                if obj not in f.keys():
                                    f[obj] = {}
                                    f[obj]['is_new_high'] = 1
                rst[d] = f
        rst_fina = {}
        dates = list(rst.keys())
        for d, v in tqdm(rst.items()):
            f = {}
            start1 = dates[(dates.index(d)) - days + 1]
            for d1 in get_calender(start=start1, end=d,region=region):
                for obj, v1 in rst[d1].items():
                    if v1['is_new_high'] >= 1:
                        if obj not in f.keys():
                            f[obj] = 1
                        else:
                            f[obj] = f[obj] + 1
            if start is not None:
                if str2datetime(d)>=str2datetime(start):
                    rst_fina[d] = f
            else:
                rst_fina[d] = f
        # if issave:
        #     save(rst_fina, os.path.join(factor_url, save_url))
        return rst_fina
class jg_holdpos:
    def update(self,name,factor_url,k1d_url,ip,start_date=None,freq="%Y%m",invest_type_nohave=[],fund_calc_mode=0,
                           top10_holdname_keys=["证券投资基金", "证券基金"], mode="0",ishalfyear=True,bypool=True,n=None,issave_detail=False,detail_name="_detail"):
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name,base_url1)
                self.calc_jgfun_holdpos(name, factor_url, ip, end=maxdate_k, start=None, invest_type_nohave=invest_type_nohave,
                           top10_holdname_keys=top10_holdname_keys, mode=mode, issave=True,bypool=bypool,n=n,
                                        issave_detail=issave_detail,detail_name=detail_name,ishalfyear=ishalfyear,
                                        fund_calc_mode=fund_calc_mode,his_run_mode=True)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            print("增量计算", maxdate,maxdate_k)
            data=self.calc_jgfun_holdpos(name, factor_url, ip, end=maxdate_k, start=maxdate, invest_type_nohave=invest_type_nohave,
                                         fund_calc_mode=fund_calc_mode,his_run_mode=False,
                                         issave_detail=issave_detail, detail_name=detail_name,ishalfyear=ishalfyear,
                           top10_holdname_keys=top10_holdname_keys, mode=mode, issave=False,bypool=bypool,n=n)

            save_new_factordata(name,data,factor_url,freq=freq)
    def deal_d0(self,objs,hold_byenddate,top10_holdname_keys,invest_type_nohave,fund_hlist,c,c0,his_run_mode=False):
        data={}
        for obj,info in tqdm(objs.items()):
            data[obj]={}
            for ann_date in info["ann_dates"]:
                t = {"fund_hold": [], "top10_floatholders": []}
                end_date1 = c.get_lastend_date(obj, ann_date)
                hold = []
                poss = []
                tt3 = []
                tt4 = []
                base_end_date = get_end_date(ann_date)
                if end_date1:
                    delta1 = str2datetime(base_end_date) - str2datetime(end_date1)
                    delta1 = delta1.days
                    if delta1 < 190:
                        for holdername, v in hold_byenddate[obj][end_date1].items():
                            for name in top10_holdname_keys:
                                if name in holdername:
                                    hold.append(v)

                for fcode, v in fund_hlist.items():
                    end_date2 = c0.get_lastend_date(fcode, ann_date)
                    if end_date2 is None:
                        continue
                    delta2 = str2datetime(base_end_date) - str2datetime(end_date2)
                    delta2 = delta2.days
                    if delta2 < 190:
                        pos = v[end_date2]
                        if obj in pos.keys():
                            i = pos[obj]
                            if str2datetime(i["ann_date"])>str2datetime(ann_date):
                                continue
                            fcode = i["ts_code"]
                            if i["invest_type"] not in invest_type_nohave:
                                poss.append(i)
                                name = i["FUND_FULLNAME"]
                                for hold0 in hold:
                                    if name in hold0["holder_name"]:
                                        if end_date2 == end_date1:
                                            if i["ann_date"] <= hold0["ann_date"]:
                                                if hold0["holder_name"] not in tt3:
                                                    tt3.append(hold0["holder_name"])
                                            else:
                                                if fcode not in tt4:
                                                    tt4.append(fcode)
                                        elif i["end_date"] > hold0["end_date"]:
                                            if hold0["holder_name"] not in tt3:
                                                tt3.append(hold0["holder_name"])
                                        elif i["end_date"] < hold0["end_date"]:
                                            if fcode not in tt4:
                                                tt4.append(fcode)
                for i in hold:
                    if i["holder_name"] not in tt3:
                        t["top10_floatholders"].append(i)
                for i in poss:
                    if i["ts_code"] not in tt4:
                        t["fund_hold"].append(i)
                data[obj][ann_date] = t
            if not his_run_mode:
                ttt = gen_tradedata_a_noobj(data[obj],info["start"], info["end"])
                data[obj] = ttt
        print("deal_d0",len(data))
        return data
    def deal_d1(self,objs,hold_byenddate,top10_holdname_keys,invest_type_nohave,fund_hlist,c,c0,c1,his_run_mode=False):
        data={}
        for obj,info in tqdm(objs.items()):
            data[obj]={}
            for ann_date in info["ann_dates"]:
                t = {"fund_hold": [], "top10_floatholders": []}
                hold = []
                poss = []
                tt3 = []
                tt4 = []
                end_date1 = c.get_lastend_date(obj, ann_date)
                if end_date1:
                    base_end_date1=get_end_date(ann_date)
                    delta1 = str2datetime(base_end_date1) - str2datetime(end_date1)
                    delta1 = delta1.days
                    if delta1 < 190:
                        for holdername, v in hold_byenddate[obj][end_date1].items():
                            for name in top10_holdname_keys:
                                if name in holdername:
                                    hold.append(v)
                base_end_date2 = get_fullreportdate(ann_date)
                for fcode, v in fund_hlist.items():
                    end_date2 = c0.get_lastend_date(fcode, ann_date)
                    if end_date2   is None:
                        continue
                    else:
                        delta2 = str2datetime(base_end_date2) - str2datetime(end_date2)
                        delta2 = delta2.days
                        if delta2 > 190:
                            continue
                    full_end_date=c1.get_lastend_date(fcode, ann_date)
                    pos = v[end_date2]
                    if full_end_date is not None:
                        if full_end_date==end_date2:
                            anndates=c1.get_anndates_byenddate(fcode, full_end_date)
                            ismerge = False
                            for i in anndates:
                                if str2datetime(i) >= str2datetime(ann_date):
                                    ismerge = True
                                    break
                            if ismerge:
                                base_end_date3 = get_fullreportdate(ann_date,-1).strftime("%Y%m%d")
                                if base_end_date3 in v.keys():
                                        for obj0, vv in v[base_end_date3].items():
                                            if obj0 not in pos.keys():
                                                pos[obj0] = vv
                        else:
                            delta2 = str2datetime(base_end_date2) - str2datetime(full_end_date)
                            delta2 = delta2.days
                            if delta2 < 190:
                                for obj0,vv in v[full_end_date].items():
                                    if obj0 not in pos.keys():
                                        pos[obj0]=vv
                    if obj in pos.keys():
                        i = pos[obj]
                        if str2datetime(i["ann_date"]) > str2datetime(ann_date):
                            continue
                        fcode = i["ts_code"]
                        if i["invest_type"] not in invest_type_nohave:
                            poss.append(i)
                            name = i["FUND_FULLNAME"]
                            for hold0 in hold:
                                if name in hold0["holder_name"]:
                                    if end_date2 == end_date1:
                                        if i["ann_date"] <= hold0["ann_date"]:
                                            if hold0["holder_name"] not in tt3:
                                                tt3.append(hold0["holder_name"])
                                        else:
                                            if fcode not in tt4:
                                                tt4.append(fcode)
                                    elif i["end_date"] > hold0["end_date"]:
                                        if hold0["holder_name"] not in tt3:
                                            tt3.append(hold0["holder_name"])
                                    elif i["end_date"] < hold0["end_date"]:
                                        if fcode not in tt4:
                                            tt4.append(fcode)
                for i in hold:
                    if i["holder_name"] not in tt3:
                        t["top10_floatholders"].append(i)
                for i in poss:
                    if i["ts_code"] not in tt4:
                        t["fund_hold"].append(i)
                data[obj][ann_date] = t
            if not his_run_mode:
                ttt=gen_tradedata_a_noobj(data[obj],info["start"],info["end"])
                data[obj]=ttt
        return data
    def gen_objs(self,keys,start,objs_info,end,iter_objs=None):
        objs={}
        if iter_objs is None:
            iter_objs=keys.keys()
        for obj in iter_objs:
            ll0 = keys[obj]
            ll0.sort()
            if start:
                if str2datetime(ll0[0]) < str2datetime(start):
                    start00 = start
                else:
                    start00 = ll0[0]
            else:
                start00 = ll0[0]
            delist_date = None
            if obj in objs_info.keys():
                delist_date = objs_info[obj]["delist_date"]
            if delist_date:
                if end:
                    if str2datetime(delist_date) < str2datetime(end):
                        end0 = delist_date
                    else:
                        end0 = end
                else:
                    end0 = end
            else:
                end0 = end
            objs[obj] = {"start": start00, "end": end0, "ann_dates": ll0}
        return objs
    def calc_holdpos_data_bypool(self,ip,fund_calc_mode=0,bypool=True,start=None,end=None,invest_type_nohave=[],
                                 top10_holdname_keys=["证券投资基金", "证券基金"],issave=True,name="jg_posdetail",
                                 factor_url=r"G:\factor\test",n=None,his_run_mode=False):
        db = getdb_client("stock", ip)
        fund_info = {}
        fund_codes = []
        objs_info={}
        if start:
            start0=str2datetime(start)-datetime.timedelta(days=300)
            find = {"ann_date": {"$gte":start0.strftime("%Y%m%d")}}
        else:
            find={}
        print("初始数据准备：fund_fullname_info，fund_info_all")
        for i in db["fund_fullname_info"].find({}):
            del i["_id"]
            fcode = i["ts_code"]
            fund_info[fcode] = i
            if i["FUND_INITIAL"] == "是":
                fund_codes.append(fcode)
        for i in db["fund_info_all"].find({}):
            del i["_id"]
            fcode = i["ts_code"]
            if fcode in fund_info.keys():
                fund_info[fcode].update(i)
            else:
                fund_codes.append(fcode)
                t = {"FUND_FULLNAME": i["name"], "FUND_INITIAL": "是"}
                t.update(i)
                fund_info[fcode] = t
        for i in db["stock_info"].find({}):
            del i["_id"]
            obj = i["obj"]
            objs_info[obj]=i
        hold_byenddate = {}
        print("获取 stock_other_top10_floatholders 去重")
        c = end_date_calcor()
        keys={}
        for i in db["stock_other_top10_floatholders"].find(find):
            del i["_id"]
            del i["ts_code"]
            obj = i["obj"]
            ann_date = str2datetime(i["ann_date"]) + datetime.timedelta(days=1)
            ann_date = ann_date.strftime("%Y%m%d")
            i["ann_date"] = ann_date
            if obj not in keys.keys():
                keys[obj] = []
            if ann_date not in keys[obj]:
                keys[obj].append(ann_date)
            end_date = i["end_date"]
            c.ondata(i)
            if obj not in hold_byenddate.keys():
                hold_byenddate[obj] = {end_date: {i["holder_name"]:i}}
            else:
                t = hold_byenddate[obj]
                if end_date not in t.keys():
                    t[end_date] = {i["holder_name"]:i}
                else:
                    if i["holder_name"] not in t[end_date].keys():
                        key = i["holder_name"]
                        t[end_date][key] = i
        c.calc_lastend_date()
        print("获取 fund_report_portfolio")
        fund_hlist = {}
        c0 = end_date_calcor(key="ts_code")
        c1 = fullend_date_calcor(key="ts_code")
        for i in db["fund_report_portfolio"].find(find,{"mkv":0,"stk_mkv_ratio":0,"stk_float_ratio":0,"obj":0,"updatetiemkey":0}):
            del i["_id"]
            if i["symbol"][-3:] in [".SH",".SZ"]:
                fcode = i["ts_code"]
                c0.ondata(i)
                c1.ondata(i)
                if fcode in fund_info.keys():
                    i["FUND_FULLNAME"] = fund_info[fcode]["FUND_FULLNAME"]
                    if "invest_type" in fund_info[fcode].keys():
                        i["invest_type"] = fund_info[fcode]["invest_type"]
                    else:
                        i["invest_type"] = ""
                else:
                    i["FUND_FULLNAME"] =""
                    i["invest_type"]=""
                if fcode not in fund_codes:
                    continue
                obj = i["symbol"][:6]
                ann_date = str2datetime(i["ann_date"])+datetime.timedelta(days=1)
                ann_date=ann_date.strftime("%Y%m%d")
                i["ann_date"]=ann_date
                end_date = i["end_date"]
                if fcode not in fund_hlist.keys():
                    fund_hlist[fcode] = {}
                if end_date not in fund_hlist[fcode].keys():
                    fund_hlist[fcode][end_date] = {}
                fund_hlist[fcode][end_date][obj]=i
                if obj not in keys.keys():
                    keys[obj] = []
                if ann_date not in keys[obj]:
                    keys[obj].append(ann_date)
        c0.calc_lastend_date()
        c1.calc_lastend_date()
        data={}
        if bypool:
            print("计算data,并行", n)
            import multiprocessing
            if n   is None:
                pool = multiprocessing.Pool(multiprocessing.cpu_count() - config0.cpu_null_n)
            else:
                pool = multiprocessing.Pool(n)
            jobs=[]
            obj_ann_date_keys=list(keys.keys())
            for iter_objs in gen_group(obj_ann_date_keys,n):
                objs=self.gen_objs(keys,start,objs_info,end,iter_objs)
                if fund_calc_mode==0:
                    p = pool.apply_async(self.deal_d0, (objs,hold_byenddate,top10_holdname_keys,invest_type_nohave,fund_hlist,c,c0,his_run_mode,))
                else:
                    p = pool.apply_async(self.deal_d1, (
                    objs, hold_byenddate, top10_holdname_keys, invest_type_nohave, fund_hlist, c, c0,c1,his_run_mode,))
                jobs.append(p)
            for p in jobs:
                data.update(p.get())
            pool.close()
            pool.join()
        else:
            print("计算data")
            objs=self.gen_objs(keys,start,objs_info,end)
            if fund_calc_mode == 0:
                data=self.deal_d0(objs, hold_byenddate, top10_holdname_keys, invest_type_nohave, fund_hlist, c, c0,his_run_mode)
            else:
                data = self.deal_d1(objs, hold_byenddate, top10_holdname_keys, invest_type_nohave, fund_hlist, c, c0,c1,his_run_mode)
        if issave:
            save_url=os.path.join(factor_url,name)
            save(data, save_url)
        db.client.close()
        return data
    def deal_stat(self,d,end0,fund_codes,mode,fund_info,ishalfyear,enddates=[]):
        data={}
        for obj,v in tqdm(d.items()):
            data[obj] = {}
            for date, v1 in v.items():
                if end0:
                    if str2datetime(date)>end0:
                        continue
                t = {"qty": 0, "floathold_ratio": 0, "freehold_ratio": 0, "jg_num": 0, "qty_noindex": 0,
                     "floathold_ratio_noindex": 0, "freehold_ratio_noindex": 0, "jg_num_noindex": 0}
                tt = []
                t1 = v1["top10_floatholders"]
                t2 = v1["fund_hold"]
                for i in t1:
                    t["qty"] = t["qty"] + i["hold_amount"]
                    t["jg_num"] = t["jg_num"] + 1
                    t["jg_num_noindex"] = t["jg_num_noindex"] + 1
                key1s = []
                key2s = []
                for i in t2:
                    fcode = i["ts_code"]
                    if mode == "0":
                        if fcode not in fund_codes:
                            continue
                        key1 = i["ts_code"] + i["end_date"] + str(i["amount"])
                        if key1 in key1s:
                            continue
                        else:
                            key1s.append(key1)
                        key2 = i["ts_code"] + i["ann_date"] + i["end_date"]
                        if key2 in key2s:
                            continue
                        else:
                            key2s.append(key2)
                    elif mode == "1":
                        key1 = i["ts_code"] + i["end_date"] + str(i["amount"])
                        if key1 in key1s:
                            continue
                        else:
                            key1s.append(key1)
                        key2 = i["ts_code"] + i["ann_date"] + i["end_date"]
                        if key2 in key2s:
                            continue
                        else:
                            key2s.append(key2)
                    elif mode == "2":
                        if fcode not in fund_codes:
                            continue
                    if ishalfyear:
                        delta = str2datetime(date) - str2datetime(i["end_date"])
                        delta = delta.days
                        if delta > 180:
                            continue
                    t["qty"] = t["qty"] + i["amount"]
                    if 'management' in fund_info[fcode].keys():
                        jg = fund_info[fcode]["management"]
                        if "invest_type" in fund_info[fcode].keys():
                            itype = fund_info[fcode]["invest_type"]
                        else:
                            itype = ""
                        if jg not in tt:
                            tt.append(jg)
                            t["jg_num"] = t["jg_num"] + 1
                            if itype != "被动指数型":
                                t["jg_num_noindex"] = t["jg_num_noindex"] + 1
                        if itype != "被动指数型":
                            t["qty_noindex"] = t["qty_noindex"] + i["amount"]
                data[obj][date] = t
        if enddates:
            data = gen_tradedata_a(data, enddates)
        return data
    def calc_jgfun_holdpos(self, name, factor_url, ip, end, start=None, invest_type_nohave=[],fund_calc_mode=0,his_run_mode=False,
                           issave_detail=False, detail_name="_detail",ishalfyear=True,
                           top10_holdname_keys=["证券投资基金", "证券基金"], mode="0",issave=False,bypool=True,n=None):
        #d = load(r"G:\factor\k1d\jg_holdpos.pkl")
        """:cvar
        1.汇总计算机构持股数量：机构为：公募+私募
        2.计算持股占流动市值的比
        3.计算机构个数
        4.计算机构个数扣除被动投资
        5.计算基金经理个数，一个私募产品算一个
        6.计算基金经理个数扣除被动投资，一个私募产品算一个
        "manager_num":0,"manager_num_noindex":0
        """
        d = self.calc_holdpos_data_bypool(ip, fund_calc_mode=fund_calc_mode, start=start, end=end,
                                          invest_type_nohave=invest_type_nohave,
                                          top10_holdname_keys=top10_holdname_keys, issave=issave_detail, name=detail_name,
                                          factor_url=factor_url, n=n, his_run_mode=his_run_mode, bypool=bypool)
        # d=load(r"G:\factor\test\jg_hold_info_detail.pkl")
        db = getdb_client("stock", ip)
        fund_info = {}
        fund_codes = []
        # size_info = load(r"G:\factor\k1d\new1\size_info.pkl")
        size_info=factordata("size_info",url=factor_url)
        for i in db["fund_fullname_info"].find({}):
            del i["_id"]
            fcode = i["ts_code"]
            fund_info[fcode] = i
            if i["FUND_INITIAL"] == "是":
                fund_codes.append(fcode)
        print("fund_codes", len(fund_codes))
        for i in db["fund_info_all"].find({}):
            del i["_id"]
            fcode = i["ts_code"]
            if fcode in fund_info.keys():
                fund_info[fcode].update(i)
            else:
                fund_codes.append(fcode)
                t={"FUND_FULLNAME":i["name"],"FUND_INITIAL":"是"}
                t.update(i)
                fund_info[fcode]=t
        obj_info={}
        for i in db["stock_info"].find({}):
            obj=i["obj"]
            if i["delist_date"]:
                obj_info[obj]=i["delist_date"]
            else:
                obj_info[obj]=end
        fund_info_manager = {}
        for i in db["fund_info_manager"].find({}):
            del i["_id"]
            fcode = i["ts_code"]
            if fcode in fund_info.keys():
                if fcode not in fund_info_manager:
                    fund_info_manager[fcode] = []
                fund_info_manager[fcode].append(i)
        data = {}
        rst={}
        print("fund_info_manager")
        end0=None
        if end:
            end0=str2datetime(end)
        print("统计持有数据data")
        if his_run_mode:
            print("计算data,并行", n)
            import multiprocessing
            if n is None:
                pool = multiprocessing.Pool(multiprocessing.cpu_count() - config0.cpu_null_n)
            else:
                pool = multiprocessing.Pool(n)
            jobs=[]
            obj_ann_date_keys=list(d.keys())
            for iter_objs in gen_group(obj_ann_date_keys, n):
                d0={}
                enddates={}
                for obj in iter_objs:
                    d0[obj]=d[obj]
                    if obj not in obj_info.keys():
                        enddates[obj]=end
                    else:
                        enddates[obj]=obj_info[obj]
                p = pool.apply_async(self.deal_stat, (d0, end0, fund_codes, mode, fund_info,ishalfyear,enddates,))
                jobs.append(p)
            for p in jobs:
                for date,v in p.get().items():
                    if date not in rst.keys():
                        rst[date]=v
                    else:
                        rst[date].update(v)
            pool.close()
            pool.join()
        else:
            data=self.deal_stat(d,end0,fund_codes,mode,fund_info,ishalfyear)
            rst = convert_byobj2bydate(data)
        print("统计持有数据完成，gen_tradedata_a")
        print("计算占有百分比")
        if his_run_mode:
            print("计算data,并行", n)
            import multiprocessing
            if n   is None:
                pool = multiprocessing.Pool(multiprocessing.cpu_count() - config0.cpu_null_n)
            else:
                pool = multiprocessing.Pool(n)
            jobs=[]
            obj_ann_date_keys=list(rst.keys())
            for iter_dates in gen_group(obj_ann_date_keys, n):
                rst0={}
                for date in iter_dates:
                    rst0[date]=rst[date]
                p = pool.apply_async(self.deal_stat_rate,(rst0,start,size_info))
                jobs.append(p)
            for p in jobs:
                data.update(p.get())
            pool.close()
            pool.join()
        else:
            data=self.deal_stat_rate(rst,start,size_info)
        if issave:
            save_url = os.path.join(factor_url, name)
            save(data, save_url)
        db.client.close()
        print("计算完成")
        return data
    def deal_stat_rate(self,rst,start,size_info):
        presize = {}
        data = {}
        if start:
            start0 = str2datetime(start)
        for date,v in tqdm(rst.items()):
            if start:
                if str2datetime(date) <start0:
                    continue
            data[date] = {}
            size_info0=size_info.get_bydate(date)
            if size_info0:
                for obj, t in v.items():
                    if obj in size_info0.keys():
                        t["floathold_ratio"] = t["qty"] / size_info0[obj]["float_share"] / 10000
                        t["freehold_ratio"] = t["qty"] / size_info0[obj]["free_share"] / 10000
                        t["floathold_ratio_noindex"] = t["qty_noindex"] / size_info0[obj]["float_share"] / 10000
                        t["freehold_ratio_noindex"] = t["qty_noindex"] / size_info0[obj]["free_share"] / 10000
                        presize[obj] = size_info0[obj]
                    elif obj in presize.keys():
                        t["floathold_ratio"] = t["qty"] / presize[obj]["float_share"] / 10000
                        t["freehold_ratio"] = t["qty"] / presize[obj]["free_share"] / 10000
                        t["floathold_ratio_noindex"] = t["qty_noindex"] / presize[obj]["float_share"] / 10000
                        t["freehold_ratio_noindex"] = t["qty_noindex"] / presize[obj]["free_share"] / 10000
                    data[date][obj] = t
            elif presize:
                for obj, t in v.items():
                    if obj in presize.keys():
                        t["floathold_ratio"] = t["qty"] / presize[obj]["float_share"] / 10000
                        t["freehold_ratio"] = t["qty"] / presize[obj]["free_share"] / 10000
                        t["floathold_ratio_noindex"] = t["qty_noindex"] / presize[obj]["float_share"] / 10000
                        t["freehold_ratio_noindex"] = t["qty_noindex"] / presize[obj]["free_share"] / 10000
                    data[date][obj] = t
        return data
class trade_days:
    def __init__(self,name="trade_days"):
        self.name=name
    def update(self,factor_url,k1d_url,ip,start_date=None,freq="%Y%m",region="C"):
        name=self.name
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                self.deal_tarde_days(factor_url,None,maxdate_k,ip=ip, name=name,issave=True,region=region)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            start_date=maxdate+datetime.timedelta(days=1)
            print("增量计算", maxdate,start_date,maxdate_k)
            factort = factordata(name,url=factor_url)
            have=factort.get_bydate(maxdate)
            data=self.deal_tarde_days(factor_url,start_date=start_date,end_date=maxdate_k,ip=ip, name=name,issave=False,have=have,region=region)
            save_new_factordata(name,data,factor_url,freq=freq)
    def deal_tarde_days(self,factor_url,start_date=None,end_date=None,ip="127.0.0.1", name="trade_days",issave=False,have=None,region="C"):
        db = getdb_client("stock", ip)
        rst = {}
        objs_delist = []
        t0 = datetime.datetime(2000, 1, 4)
        for i in db["stock_info"].find({}):
            objs_delist.append(i)
        for i in tqdm(objs_delist):
            if "(测试)" in i["name"]:
                continue
            obj = i["obj"]
            start = i["list_date"]
            end = i["delist_date"]
            if end_date is not None:
                if end:
                    if str2datetime(end)>str2datetime(end_date):
                        end=str2datetime(end_date)
                else:
                    end=str2datetime(end_date)
            if end:
                if str2datetime(end) > t0:
                    t1 = datetime.datetime.strptime(start, "%Y%m%d")
                    if start_date:
                        if obj not in have.keys():
                            n=0
                        else:
                            n=have[obj]
                        start=str2datetime(start_date)
                    else:
                        if t1 < t0:
                            n = t0 - t1
                            n = n.days
                        else:
                            n = 0
                    for d in get_calender(start, end,region=region):
                        n = n + 1
                        if d not in rst.keys():
                            t = {obj: n}
                            rst[d] = t
                        else:
                            t = rst[d]
                            t[obj] = n
            else:
                t1 = datetime.datetime.strptime(start, "%Y%m%d")
                if t1 < t0:
                    n = t0 - t1
                    n = n.days
                else:
                    n = 0
                for d in get_calender(start, end,region=region):
                    n = n + 1
                    if d not in rst.keys():
                        t = {obj: n}
                        rst[d] = t
                    else:
                        t = rst[d]
                        t[obj] = n
        if issave:
            save_url=os.path.join(factor_url,name)
            save(rst, save_url)
        return rst
class is_st:#数据源有问题，暂时弃用
    def __init__(self,name="is_st"):
        self.name=name
    def update(self,factor_url,k1d_url,ip,start_date=None,freq="%Y%m",region="C"):
        name=self.name
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                self.deal_is_st(name,factor_url,ip,start="20060411",end_date=maxdate_k,issave=True,region=region)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None

            print("增量计算", maxdate,maxdate_k)
            factort = factordata("is_st",url=factor_url)
            have=factort.get_bydate(maxdate)
            data=self.get_is_st_bydate(ip,have,start_date=maxdate,end_date=maxdate_k,region=region)
            save_new_factordata(name,data,factor_url,freq=freq)
    def get_is_st_bydate(self,ip,have,start_date="20060411",end_date=None,region="C"):
        db = getdb_client("stock", ip)
        objs_delist = {}
        for i in db["stock_info"].find({}):
            obj = i["obj"]
            objs_delist[obj] = i
        start_date=str2datetime(start_date)
        start=start_date.strftime("%Y%m%d")
        rst={}
        for obj in have:
            rst[obj] = {"sart":start,"end":None}
        for i in db["stock_info_name_change"].find({"start_date":{"$gte":start}}):
            if "撤销ST" in i["change_reason"]:
                if obj not in rst.keys():
                    rst[obj] = {}
                end0 = i["start_date"]
                rst[obj]["end"] = end0
            if "ST" in i["name"]:
                obj = i["obj"]
                if obj not in rst.keys():
                    rst[obj] = {}
                if i["start_date"] >= start:
                    start0 = i["start_date"]
                else:
                    start0 = start
                rst[obj]["start"] = start0
                if i["end_date"]:
                    end0 = i["end_date"]
                elif obj in objs_delist.keys():
                    end0 = objs_delist[obj]["delist_date"]
                elif end_date is not None:
                    end0=end_date
                else:
                    end0 = None
                rst[obj]["end"] = end0
        rst1 = {}
        for obj, v in rst.items():
            if "start" in v.keys():
                if v["end"]   is None:
                    for d in get_calender(v["start"], v["end"],region=region):
                        if d not in rst1.keys():
                            rst1[d] = [obj]
                        else:
                            rst1[d].append(obj)
                else:
                    if str2datetime(v["end"]) > str2datetime(v["start"]):
                        for d in get_calender(v["start"], v["end"],region=region):
                            if d not in rst1.keys():
                                rst1[d] = [obj]
                            else:
                                rst1[d].append(obj)
        db.client.close()
        return rst1
    def deal_is_st(self,name,factor_url,ip,start="20060411",end_date=None,issave=False,region="C"):
        db = getdb_client("stock", ip)
        rst = {}
        objs_delist = {}
        for i in db["stock_info"].find({}):
            obj = i["obj"]
            objs_delist[obj] = i
        for i in db["stock_info_name_change"].find({}).sort([("start_date",1)]):
            obj = i["obj"]
            if obj not in rst.keys():
                rst[obj] = []
            rst[obj].append(i)
        rst1={}
        for obj,v in rst.items():
            t=[]
            prestart=None
            if obj not in rst1.keys():
                rst1[obj]=[]
            for i in v:
                if "ST" in i["name"]:
                    if len(t)==0:
                        if prestart is not None:
                            if i["start_date"]>prestart:
                                t.append(i["start_date"])
                        else:
                            t.append(i["start_date"])
                else:
                    if len(t)==1:
                        t.append(i["start_date"])
                        rst1[obj].append(deepcopy(t))
                        t=[]
                prestart=i["start_date"]
            if len(t)==1:
                t.append(end_date)
                rst1[obj].append(deepcopy(t))
        rst2={}
        for obj ,v in rst1.items():
            for i in v:
                for d in get_calender(i[0], i[1],region=region):
                    if d not in rst2.keys():
                        rst2[d] = [obj]
                    else:
                        rst2[d].append(obj)
        db.client.close()
        if issave:
            save_url=os.path.join(factor_url, name)
            save(rst2,save_url)
        return rst2
class index_detail_byday:
    def update(self, name,factor_url, k1d_url,ip,start_date=None, freq="%Y%m",end_date=None,level="L2",mode="0",cu_level="index_calssify_sw_level",cu_detail="index_calssify_sw_detail",region="C"):
        from jili.data.db import tradecal as tcl
        print("update", name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                self.get_index_detail_byday(factor_url,name,ip,start_date=start_date,end_date=maxdate_k,level=level,mode=mode,cu_level=cu_level,cu_detail=cu_detail,region=region)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            start_date = tcl.get_diff_tradeday(maxdate, 1,region=region)
            print("增量计算", maxdate, start_date, maxdate_k)
            factort = factordata(name,url=factor_url)
            init_detail=factort.get_bydate(maxdate)
            data = self.get_index_detail_bydate(ip,init_detail,start_date=start_date,end_date=maxdate_k,level=level,mode=mode,cu_level=cu_level,cu_detail=cu_detail,region=region)
            save_new_factordata(name, data, factor_url, freq=freq)
    def get_index_detail_bydate0(self,ip,init_detail,start_date=None,end_date=None,level="L2",mode="0",cu_level="index_calssify_sw_level",cu_detail="index_calssify_sw_detail",region="C"):
        db = getdb_client("stock", ip)
        cu=db[cu_detail]
        sw={}
        rst = {}
        detail = {}
        for i in db[cu_level].find({"level":level}):
            id=i["index_code"]
            if "index_name" in i.keys():
                sw[id] = i["index_name"]
            else:
                sw[id]=i["industry_name"]
        min_date = None
        init_date = str2datetime(start_date)
        if mode=="0":
            start_date=init_date.strftime("%Y-%m-%d")
            for i in cu.find({"date":{"$gte":start_date}}):
                del i["_id"]
                index = i["index_code"]
                date=i["date"]
                if index in sw.keys():
                    if index not in detail.keys():
                        detail[index]={}
                    if date not in detail[index].keys():
                        detail[index][date]=[]
                    detail[index][date].append(i)
                    # indate = "date"
            #         if min_date:
            #             if min_date > i[indate]:
            #                 min_date = i[indate]
            #         else:
            #             min_date = i[indate]
            # if start_date:
            #     if str2datetime(min_date) < str2datetime(start_date):
            #         min_date = start_date
            detaild = {}
            for index_code, v in init_detail.items():
                detaild[index_code] = {start_date: deepcopy(v)}
            for index_code, v in detail.items():
                l=list(v.keys())
                l.sort()
                if index_code in init_detail.keys():
                    ll=deepcopy(init_detail[index_code])
                else:
                    ll = []
                for date in l:
                    l1 = []
                    for i in v[date]:
                        action = i["action"]
                        obj = i["obj"]
                        if action == "纳入":
                            if obj not in ll:
                                ll.append(obj)
                        elif action == "剔除":
                            if obj not in l1:
                                l1.append(obj)
                    for i in l1:
                        if i not in ll:
                            print(index_code,date,i,"无纳入")
                        else:
                            ll.remove(i)
                    if index_code not in detaild.keys():
                        detaild[index_code]={}
                    detaild[index_code][date]=deepcopy(ll)

            rst=gen_tradedata_a(detaild,end_date)
        else:
            detail=deepcopy(init_detail)
            start_date=str2datetime(start_date).strftime("%Y-%m-%d")
            for i in cu.find({{"$or":[{"in_date":{"$gte":start_date}},{"out_date":{"$gte":start_date}}]}}):
                del i["_id"]
                index = i["index_code"]
                if index in sw.keys():
                    if index in detail.keys():
                        detail[index].append(i)
                    else:
                        detail[index] = [i]
                    if "in_date" not in i.keys():
                        indate = "date"
                    else:
                        indate = "in_date"

                    if min_date:
                        if str2datetime(min_date) > str2datetime(i[indate]):
                            min_date = i[indate]
                    else:
                        min_date = i[indate]
            if start_date:
                if str2datetime(min_date) < str2datetime(start_date):
                    min_date = start_date
            for d in tqdm(get_calender(start=min_date,end=end_date,region=region)):
                date=d.strftime("%Y%m%d")
                for index_code,v in detail.items():
                    if index_code in init_detail.keys():
                        ll = init_detail[index_code]
                    else:
                        ll = []
                    for i in v:
                        in_date=i["in_date"]
                        out_date=i["out_date"]
                        obj=i["obj"]
                        if out_date:
                            if in_date<=date and date<=out_date:
                                if obj not in ll:
                                    ll.append(obj)
                        else:
                            if in_date<=date:
                                if obj not in ll:
                                    ll.append(obj)
                    if ll:
                        if d not in rst.keys():
                            rst[d]={index_code:ll}
                        else:
                            t=rst[d]
                            t[index_code]=ll
        return rst
    def get_index_detail_bydate(self,ip,init_detail={},start_date=None,end_date=None,level="L2",mode="0",cu_level="index_calssify_sw_level",cu_detail="index_calssify_sw_detail",region="C"):
        db = getdb_client("stock", ip)
        cu=db[cu_detail]
        sw={}
        rst = {}
        detail = {}
        for i in db[cu_level].find({"level":level}):
            id=i["index_code"]
            if "index_name" in i.keys():
                sw[id] = i["index_name"]
            else:
                sw[id]=i["industry_name"]
        min_date = None
        if mode=="0":
            for i in cu.find({}):
                del i["_id"]
                index = i["index_code"]
                date=str2datetime(i["date"])
                if index in sw.keys():
                    if index not in detail.keys():
                        detail[index]={}
                    if date not in detail[index].keys():
                        detail[index][date]=[]
                    detail[index][date].append(i)
                    indate = "date"
                    if min_date:
                        if str2datetime(min_date) > str2datetime(i[indate]):
                            min_date = i[indate]
                    else:
                        min_date = i[indate]
            if start_date:
                if str2datetime(min_date) < str2datetime(start_date):
                    min_date = start_date
            detaild={}
            for index_code, v in detail.items():
                l=list(v.keys())
                l.sort()
                ll = []
                for date in l:
                    l1 = []
                    for i in v[date]:
                        action = i["action"]
                        obj = i["obj"]
                        if action == "纳入":
                            if obj not in ll:
                                ll.append(obj)
                        elif action == "剔除":
                            if obj not in l1:
                                l1.append(obj)
                    for i in l1:
                        if i in ll:
                            # print(index_code,date,i,"无纳入")
                        # else:
                            ll.remove(i)
                    if index_code not in detaild.keys():
                        detaild[index_code]={}
                    detaild[index_code][date]=deepcopy(ll)
            rst=gen_tradedata_a(detaild,end_date,start_date)
        else:
            for i in cu.find({}):
                del i["_id"]
                index = i["index_code"]
                if index in sw.keys():
                    if index in detail.keys():
                        detail[index].append(i)
                    else:
                        detail[index] = [i]
                    if "in_date" not in i.keys():
                        indate = "date"
                    else:
                        indate = "in_date"

                    if min_date:
                        if str2datetime(min_date) > str2datetime(i[indate]):
                            min_date = i[indate]
                    else:
                        min_date = i[indate]
            if start_date:
                if str2datetime(min_date) < str2datetime(start_date):
                    min_date = start_date
            for d in tqdm(get_calender(start=start_date,end=end_date,region=region)):
                date=d.strftime("%Y%m%d")
                for index_code,v in detail.items():
                    ll=[]
                    for i in v:
                        in_date=i["in_date"]
                        out_date=i["out_date"]
                        obj=i["obj"]
                        if out_date:
                            if in_date<=date and date<=out_date:
                                if obj not in ll:
                                    ll.append(obj)
                        else:
                            if in_date<=date:
                                if obj not in ll:
                                    ll.append(obj)
                    if ll:
                        if d not in rst.keys():
                            rst[d]={index_code:ll}
                        else:
                            t=rst[d]
                            t[index_code]=ll
        return rst
    def get_index_detail_byday(self,factor_url,name,ip,start_date=None,end_date=None,level="L2",mode="0",cu_level="index_calssify_sw_level",cu_detail="index_calssify_sw_detail",region="C"):
        db = getdb_client("stock", ip)
        cu=db[cu_detail]
        sw={}
        rst = {}
        detail = {}
        for i in db[cu_level].find({"level":level}):
            id=i["index_code"]
            if "index_name" in i.keys():
                sw[id] = i["index_name"]
            else:
                sw[id]=i["industry_name"]
        min_date = None
        if mode=="0":
            for i in cu.find({}):
                del i["_id"]
                index = i["index_code"]
                date=str2datetime(i["date"])
                if index in sw.keys():
                    if index not in detail.keys():
                        detail[index]={}
                    if date not in detail[index].keys():
                        detail[index][date]=[]
                    detail[index][date].append(i)
                    indate = "date"
                    if min_date:
                        if str2datetime(min_date) > str2datetime(i[indate]):
                            min_date = i[indate]
                    else:
                        min_date = i[indate]
            if start_date:
                if str2datetime(min_date) < str2datetime(start_date):
                    min_date = start_date
            detaild={}
            for index_code, v in detail.items():
                l=list(v.keys())
                l.sort()
                ll = []
                for date in l:
                    l1 = []
                    for i in v[date]:
                        action = i["action"]
                        obj = i["obj"]
                        if action == "纳入":
                            if obj not in ll:
                                ll.append(obj)
                        elif action == "剔除":
                            if obj not in l1:
                                l1.append(obj)
                    for i in l1:
                        if i not in ll:
                            print(index_code,date,i,"无纳入")
                        else:
                            ll.remove(i)
                    if index_code not in detaild.keys():
                        detaild[index_code]={}
                    detaild[index_code][date]=deepcopy(ll)
            rst=gen_tradedata_a(detaild,end_date)
        else:
            for i in cu.find({}):
                del i["_id"]
                index = i["index_code"]
                if index in sw.keys():
                    if index in detail.keys():
                        detail[index].append(i)
                    else:
                        detail[index] = [i]
                    if "in_date" not in i.keys():
                        indate = "date"
                    else:
                        indate = "in_date"

                    if min_date:
                        if str2datetime(min_date) > str2datetime(i[indate]):
                            min_date = i[indate]
                    else:
                        min_date = i[indate]
            if start_date:
                if str2datetime(min_date) < str2datetime(start_date):
                    min_date = start_date
            for d in tqdm(get_calender(start=min_date,end=end_date,region=region)):
                date=d.strftime("%Y%m%d")
                for index_code,v in detail.items():
                    ll=[]
                    for i in v:
                        in_date=i["in_date"]
                        out_date=i["out_date"]
                        obj=i["obj"]
                        if out_date:
                            if in_date<=date and date<=out_date:
                                if obj not in ll:
                                    ll.append(obj)
                        else:
                            if in_date<=date:
                                if obj not in ll:
                                    ll.append(obj)
                    if ll:
                        if d not in rst.keys():
                            rst[d]={index_code:ll}
                        else:
                            t=rst[d]
                            t[index_code]=ll
        save_url = os.path.join(factor_url, name)
        save(rst,save_url)
        return rst
class new_high:
    def update(self,name,factor_url,k1d_url,args={},freq="%Y%m",start_date=None):
        from jili.data.stocklocal import iter_bardict
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        url_byobj = os.path.join(kbase_url, "byobj")
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        print("update", name)
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                self.deal_new_high_days(url_byobj,factor_url, name=name, args=args,freq=freq)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            print("增量计算", maxdate,maxdate_k)
            kk={}
            for date,v in iter_bardict(start=maxdate,end=maxdate_k,url=k1d_url,ktype="k1d"):
                for obj,bar in v.items():
                    if obj not in kk.keys():
                        kk[obj]=[]
                    kk[obj].append(bar)
            rst={}
            a=factordata(name,url=factor_url)
            data=a.get_bydate(maxdate)
            data0={}
            for obj,v in kk.items():
                r=self.calc_new_high_days(v, args)
                if obj not in data.keys():
                    for d,vv in r.items():
                        if d not in data0.keys():
                            data0[d]={}
                        if obj not in data0[d].keys():
                            data0[d][obj]=vv
                else:
                    ll=list(r.keys())
                    ll.sort()
                    vv={}
                    isflag0=True
                    for date in ll[1:]:
                        vt=r[date]
                        if vt==0:
                            isflag0=False
                        if isflag0:
                            vvv=vt+data[obj]
                        else:
                            vvv = vt
                        if date not in data0.keys():
                            data0[date]={}
                        if obj not in data0[date].keys():
                            data0[date][obj]=vvv
            save_new_factordata(name,data0,factor_url,freq=freq)
    # 创新高天数
    def deal_new_high_days(self,dataurl,factor_url, name="new_high_days", args={},freq="%Y%m"):
        rst = {}
        for i in tqdm(os.listdir(dataurl)):
            obj = i.split("_")[0]
            k = load(os.path.join(dataurl, i))
            r = self.calc_new_high_days(k, args)
            for d, v in r.items():
                month_str=d.strftime(freq)
                if month_str not in rst.keys():
                    rst[month_str]={}
                if d not in rst[month_str].keys():
                    rst[month_str][d] = {}
                rst[month_str][d][obj] = v
        # save(rst, os.path.join(factor_url,name))
        for month_str,v in rst.items():
            save(v, os.path.join(factor_url, "bydate",name,month_str+".pkl"))
        return rst
    def calc_new_high_days(self,k, args={}):
        """
        k:list
        """
        price = args.get("price", "close")
        rst = {}
        if len(k) == 1:
            for i in range(len(k)):
                t = k[i]
                date = t["timekey"]
                rst[date] = 0
        else:
            for i in range(len(k)):
                t = k[i]
                date = t["timekey"]
                k1 = k[:i]
                n = 0
                for tt in reversed(k1):
                    if t[price] > tt[price]:
                        n = n + 1
                    else:
                        break
                rst[date] = n
        return rst
class rps:
    def update(self,name,factor_url,k1d_url,pctchg,start_date=None,freq="%Y%m",region="C"):
        from jili.data.db import tradecal as tcl
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                self.deal_his(name,factor_url,pctchg,start=None,issave=True)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            start_date = tcl.get_diff_tradeday(maxdate, 1,region=region)
            print("增量计算", maxdate,start_date,maxdate_k)
            data=self.deal_his(name,factor_url,pctchg,start=start_date,issave=False)
            save_new_factordata(name,data,factor_url,freq=freq)
    def deal_his(self,name,factor_url,pctchg,start=None,issave=False):
        rst = {}
        a=factordata(pctchg,url=factor_url)
        if start is None:
            data=a.get_all()
            for d, v in data.items():
                vd = pd.DataFrame([v], index=[pctchg]).T
                vd.dropna(inplace=True)
                vd1 = vd
                # vd1=vd.loc[not np.isnan(vd[pctchg])]
                vd1.sort_values([pctchg], ascending=False, inplace=True)
                vd1["name"] = list(range(len(vd1), 0, -1))
                vd1["name"] = 100 * vd1["name"] / len(vd1)
                # vd1.index=vd1["obj"]
                rst[d] = vd1["name"].to_dict("index")
        else:
            for d, v in a.iter_data(start=start):
                vd = pd.DataFrame([v], index=[pctchg]).T
                vd.dropna(inplace=True)
                vd1 = vd
                # vd1=vd.loc[not np.isnan(vd[pctchg])]
                vd1.sort_values([pctchg], ascending=False, inplace=True)
                vd1["name"] = list(range(len(vd1), 0, -1))
                vd1["name"] = 100 * vd1["name"] / len(vd1)
                # vd1.index=vd1["obj"]
                rst[d] = vd1["name"].to_dict("index")
        if issave:
            save_url=os.path.join(factor_url,name)
            save(rst, save_url)
        return rst
class base_factor:
    def __init__(self,name):
        self.name=name
    def getk1d_maxdate(self,k1d_url):
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        return maxdate_k
    def update(self,**args):
        factor_url=args.get("factor_url")
        k1d_url = args.get("k1d_url")
        freq = args.get("freq","%Y%m")
        start_date = args.get("start_date",None)
        print("update",self.name)
        base_url = os.path.join(factor_url, "bydate", self.name)
        base_url1 = os.path.join(factor_url, self.name + ".pkl")
        maxdate_k=self.getk1d_maxdate(k1d_url=k1d_url)
        isadd = True
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", self.name)
                factor_data = self.calc_data(args)
                self.save_factordata(factor_data, args)
            else:
                print("因子存在,按日期清洗", freq, self.name)
                factort = factordata("test")
                factort.deal2bydate(factor_url, [self.name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(self.name, "k线与因子已经是最新", maxdate)
                return None
            print("增量计算", maxdate,maxdate_k)
            args["start_date"]=maxdate
            factor_data=self.calc_data(args)
            self.save_factordata(factor_data,args)
    def save_factordata(self,data,args):
        factor_url = args.get("factor_url")
        freq = args.get("freq", "%Y%m")
        if data:
            url = os.path.join(factor_url, self.name + ".pkl")
            if os.path.exists(url):
                a = load(url)
                a.update(data)
                save(a, url)
            else:
                save(data, url)
            data1 = {}
            for date, v in data.items():
                key = date.strftime(freq)
                if key not in data1.keys():
                    data1[key] = {}
                data1[key][date] = v
            for key, v in data1.items():
                save_url = os.path.join(factor_url, "bydate", self.name, key + ".pkl")
                if os.path.exists(save_url):
                    a = load(save_url)
                    a.update(v)
                    save(a, save_url)
                else:
                    save(v, save_url)
    def calc_data(self,args):
        """:args
        继承必须实现
        必要参数：
        factor_url
        k1d_url
        可选参数：
        freq
        start_date
        默认db转factor
        必须：
        ip
        db
        cu
        trade_date
        date_format
        keys:date以下的key 比如obj 等
        """
        data={}
        ip=args.get("ip")
        dbname=args.get("dbname")
        cu=args.get("cu")
        trade_date=args.get("trade_date")
        start_date=args.get("start_date",None)
        end_date = args.get("end_date", None)
        key=args.get("key",None)
        values=args.get("values", {})
        find={}
        if start_date:
            if end_date:
                date_format=args.get("date_format",None)
                if date_format:
                    start_date=start_date.strftime(date_format)
                    end_date = end_date.strftime(date_format)
                find[trade_date]={"$gte":start_date,"$lte":end_date}
            else:
                date_format=args.get("date_format",None)
                if date_format:
                    start_date=start_date.strftime(date_format)
                find[trade_date]={"$gte":start_date}
        db=getdb_client(dbname=dbname,ip=ip)
        if values:
            curr=db[cu].find(find,values).sort([(trade_date,1)]).batch_size(5000)
        else:
            curr = db[cu].find(find).sort([(trade_date,1)]).batch_size(5000)
        for i in curr:
            # print(i)
            del i["_id"]
            d=str2datetime(i[trade_date])
            if d not in data.keys():
                data[d]={}
            if key:
                k=i[key]
                data[d][k]=i
            else:
                data[d]=i
        db.client.close()
        return data
class stock_base_info:
    def __init__(self,name=["trade_info","finance_info","size_info"]):
        self.name=name
    def update(self,ip,factor_url,k1d_url,freq="%Y%m",start_date=None,region="C"):
        print("update,trade_info,finance_info,size_info")
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", "trade_info")
        isadd = True
        l = os.listdir(base_url)
        l.sort()
        data = load(os.path.join(base_url, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            factort = factordata("test")
            factort.deal2bydate(factor_url, self.name)
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(self.name, "k线与因子已经是最新", maxdate)
                return None
            print("增量计算", maxdate,maxdate_k)
            maxdate=maxdate.strftime("%Y%m%d")
            predata={}
            for name in self.name:
                l = os.listdir(os.path.join(factor_url, "bydate", name))
                l.sort()
                aa=load(os.path.join(factor_url, "bydate", name, l[-1]))
                kk = max(list(aa.keys()))
                predata[name] =aa[kk]
            trade_info,finance_info,size_info=self.calc_data(ip,factor_url,find={"trade_date":{"$gte":maxdate}},issave=False,predata=predata,region=region)
            save_new_factordata("trade_info",trade_info,factor_url,freq=freq)
            save_new_factordata("finance_info", finance_info, factor_url, freq=freq)
            save_new_factordata("size_info", size_info, factor_url, freq=freq)

    def calc_data(self,ip,location,find={},issave=False,predata={},region="C"):
        db = getdb_client("stock", ip)
        trade_info={}
        finance_info={}
        size_info={}
        objs_delist = []
        for i in db["stock_info"].find({}):
            i["start"]=str2datetime(i["list_date"])
            if i["delist_date"]:
                i["end"] = str2datetime(i["delist_date"])
            else:
                i["end"]=datetime.datetime.now()
            objs_delist.append(i)
        for i in db["stock_daily_basic"].find(find):
            d=i["trade_date"]
            d=datetime.datetime.strptime(d,"%Y%m%d")
            obj=i["obj"]
            t0 = {"close": i["close"], "turnover_rate": i["turnover_rate"], "turnover_rate_f": i["turnover_rate_f"],
                 "volume_ratio": i["volume_ratio"]}
            t1 = {"pe": i["pe"], "pe_ttm": i["pe_ttm"], "pb": i["pb"], "ps": i["ps"], "ps_ttm": i["ps_ttm"],
                  "dv_ratio": i["dv_ratio"], "dv_ttm": i["dv_ttm"]}

            t2 = {"total_share": i["total_share"], "float_share": i["float_share"], "free_share": i["free_share"],
                  "total_mv": i["total_mv"], "circ_mv": i["circ_mv"]}
            if d not in trade_info.keys():
                trade_info[d]={}
                trade_info[d][obj]=t0
            else:
                t=trade_info[d]
                t[obj]=t0
            if d not in finance_info.keys():
                finance_info[d]={}
                finance_info[d][obj]=t1
            else:
                t=finance_info[d]
                t[obj]=t1
            if d not in size_info.keys():
                size_info[d]={}
                size_info[d][obj]=t2
            else:
                t=size_info[d]
                t[obj]=t2
        if predata:
            t1 = predata["trade_info"]
            t2 = predata["finance_info"]
            t3 = predata["size_info"]
        else:
            t1 = {}
            t2 = {}
            t3 = {}
        ds=list(trade_info.keys())
        ds.sort()
        for d in tqdm(get_calender(ds[0], ds[-1],region=region)):
            if d not in trade_info.keys():
                trade_info[d]=deepcopy(t1)
            else:
                t=trade_info[d]
                for i in objs_delist:
                    obj=i["obj"]
                    if d >=i["start"] and d<=i["end"]:
                        if obj not in t.keys():
                            if obj  in t1.keys():
                                t[obj]=deepcopy(t1[obj])
                            else:
                                print(obj,d)
                        else:
                            t1[obj]=t[obj]
            if d not in finance_info.keys():
                finance_info[d]=deepcopy(t2)
            else:
                t=finance_info[d]
                for i in objs_delist:
                    obj=i["obj"]
                    if d >=i["start"] and d<=i["end"]:
                        if obj not in t.keys():
                            if obj  in t2.keys():
                                t[obj]=deepcopy(t2[obj])
                            else:
                                print(obj,d)
                        else:
                            t2[obj]=t[obj]
            if d not in size_info.keys():
                size_info[d]=deepcopy(t3)
            else:
                t=size_info[d]
                for i in objs_delist:
                    obj=i["obj"]
                    if d >=i["start"] and d<=i["end"]:
                        if obj not in t.keys():
                            if obj  in t3.keys():
                                t[obj]=deepcopy(t3[obj])
                            else:
                                print(obj,d)
                        else:
                            t3[obj]=t[obj]
        if issave:
            save(trade_info,os.path.join(location,"trade_info"))
            save(finance_info, os.path.join(location, "finance_info"))
            save(size_info, os.path.join(location, "size_info"))
        return trade_info,finance_info,size_info
class index_score1:
    def update(self,name,factor_url,k1d_url,
               base_facotrs=["is_wind_new_theme_detail","trade_days","jg_hold_info", "rps_50",'ma_amount_20','size_info'],
               start_date=None,
               arg={"level":87,"freehold_ratio":0.02,"trade_days":180,'ma_amount_20':1.8e8,'size_info':10000*50},
               freq="%Y%m"):
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                self.calc_index_strength(name,factor_url,base_facotrs=base_facotrs,start_date=None,end_date=None,arg=arg, issave=True)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            start_date=maxdate+datetime.timedelta(days=1)
            print("增量计算", maxdate,start_date,maxdate_k)
            data=self.calc_index_strength(name,factor_url,base_facotrs=base_facotrs,start_date=maxdate,end_date=maxdate_k,arg=arg, issave=False)
            save_new_factordata(name,data,factor_url,freq=freq)
    def calc_index_strength(self,name,factor_url,
                            base_facotrs=["is_wind_new_theme_detail","trade_days","jg_hold_info", "rps_50",'ma_amount_20','size_info'],
                            start_date=None,end_date=None,
                            arg={"level":87,"freehold_ratio":0.02,"trade_days":180,'ma_amount_20':1.8e8,'size_info':10000*50},
                            issave=False):
        f1 = factordata(base_facotrs[3], url=factor_url)
        f2 = factordata(base_facotrs[1], url=factor_url)
        f3 = factordata(base_facotrs[0], url=factor_url)
        f4 = factordata(base_facotrs[2], url=factor_url)
        f5 = factordata(base_facotrs[4], url=factor_url)
        f6 = factordata(base_facotrs[5], url=factor_url)
        if start_date:
            rpsd = {}
            trade_days = {}
            detaild = {}
            top10_holders_jgrate = {}
            ma_amount_20 = {}
            size_info = {}
            for k, v in f1.iter_data(start_date, end_date):
                rpsd[k] = v
            for k, v in f2.iter_data(start_date, end_date):
                trade_days[k] = v
            for k, v in f3.iter_data(start_date, end_date):
                detaild[k] = v
            for k, v in f4.iter_data(start_date, end_date):
                top10_holders_jgrate[k] = v
            for k, v in f5.iter_data(start_date,end_date):
                ma_amount_20[k] = v
            for k, v in f6.iter_data(start_date,end_date):
                size_info[k] = v
        else:
            rpsd = f1.get_all()
            trade_days = f2.get_all()
            detaild = f3.get_all()
            top10_holders_jgrate = f4.get_all()
            ma_amount_20 = f5.get_all()
            size_info = f6.get_all()
        # rpsd = load(os.path.join(r"G:\factor\k1d", arg[3]))
        # trade_days = load(os.path.join(r"G:\factor\k1d", arg[1]))
        # detaild = load(os.path.join(r"G:\factor\k1d", arg[0]))
        # top10_holders_jgrate = load(os.path.join(r"G:\factor\k1d", arg[2]))
        rst = {}
        for d, v in tqdm(detaild.items()):
            if (d in rpsd.keys()) & (d in trade_days.keys()):
                if d not in rst.keys():
                    rst[d] = {}
                t = rst[d]
                for index, objs in v.items():
                    topn = 0
                    botn = 0
                    for obj in objs:
                        if obj in ma_amount_20[d].keys():
                            if ma_amount_20[d][obj]   is not None:
                                if ma_amount_20[d][obj] >= arg["ma_amount_20"]:
                                    if obj in trade_days[d].keys():
                                        if trade_days[d][obj] >= arg["trade_days"]:
                                            if obj in top10_holders_jgrate[d].keys():
                                                if top10_holders_jgrate[d][obj]["freehold_ratio"] >= arg["freehold_ratio"]:
                                                    if obj in size_info[d].keys():
                                                        if size_info[d][obj]['total_mv'] >= arg['size_info']:
                                                            botn = botn + 1
                                                            if obj in rpsd[d].keys():
                                                                if rpsd[d][obj] >= arg['level']:
                                                                    topn = topn + 1
                    if botn > 0:
                        t0 = topn ** 2 / botn
                    else:
                        t0 = 0
                    t[index] = t0
        if issave:
            save_url=os.path.join(factor_url,name)
            save(rst,save_url)
        return rst

class MA120H:
    # close/ma(close,120)
    def update(self,name,factor_url,k1d_url,close,ma120,start_date=None,freq="%Y%m",region="C"):
        from jili.data.db import tradecal as tcl
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                self.deal_his(name,factor_url,close,ma120,start=None,issave=True)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            start_date = tcl.get_diff_tradeday(maxdate, 1,region=region)
            print("增量计算", maxdate,start_date,maxdate_k)
            data=self.deal_his(name,factor_url,close,ma120,start=start_date,issave=False)
            save_new_factordata(name,data,factor_url,freq=freq)
    def deal_his(self,name,factor_url,close,ma120,start=None,issave=False):
        rst = {}
        f1 = factordata(close, url=factor_url)
        f2=factordata(ma120,url=factor_url)
        if start:
            a1={}
            a2={}
            for k,v in f1.iter_data(start):
                a1[k]=v
            for k,v in f2.iter_data(start):
                a2[k] = v
        else:
            a1 = f1.get_all()
            a2 = f2.get_all()

        for d, v in tqdm(a2.items()):
            f = {}
            for obj, v1 in v.items():
                if v1:
                    if obj in a1[d].keys():
                        if obj not in f.keys():
                            f[obj] = {}
                            f[obj] = a1[d][obj] / v1
            rst[d] = f
        if issave:
            save_url=os.path.join(factor_url,name)
            save(rst, save_url)
        return rst

class stock_fina_indicator:
    def update(self,ip,factor_url,k1d_url,freq="%Y%m",start_date=None,region="C"):
        from jili.data.db import tradecal as tcl
        print("update", 'stock_fina_indicator')
        db = getdb_client("stock", ip)
        for i in db['stock_fina_indicator'].find():
            factor_list = list(i.keys())
            break
        factor_list = ['fina_'+i for i in factor_list if i not in ['_id','ts_code','ann_date','end_date','obj']]
        fina_indicator, c, objs_delist = self.load_fina_indicator(ip)
        for name in tqdm(factor_list):
            kbase_url = os.path.join(k1d_url, "hfq", "k1d")
            base_url = os.path.join(factor_url, "bydate", name)
            base_url1 = os.path.join(factor_url, name + ".pkl")
            isadd = True
            url_bymonth = os.path.join(kbase_url, "bymonth")
            l = os.listdir(url_bymonth)
            l.sort()
            data = load(os.path.join(url_bymonth, l[-1]))
            maxdate_k = max(list(data.keys()))
            if not os.path.exists(base_url):
                if not os.path.exists(base_url1):
                    isadd = False
                    print("因子不存在,初始计算", name)
                    self.calc_data(name,fina_indicator,c,objs_delist, factor_url,start=None,end=None,issave=True,region=region)
                else:
                    print("因子存在,按日期清洗", freq, name)
                factort = factordata("test")
                factort.deal2bydate(factor_url, [name])
            if isadd:
                if start_date:
                    maxdate = str2datetime(start_date)
                else:
                    l = os.listdir(base_url)
                    l.sort()
                    data = load(os.path.join(base_url, l[-1]))
                    maxdate = max(list(data.keys()))
                if maxdate == maxdate_k:
                    print(name, "k线与因子已经是最新", maxdate)
                    return None
                print("增量计算", maxdate, maxdate_k)
                data = self.calc_data(name,fina_indicator,c,objs_delist, factor_url,start=maxdate,end = maxdate_k ,issave=False,region=region)
                save_new_factordata(name, data, factor_url, freq=freq)
    def load_fina_indicator(self,ip):
        db = getdb_client("stock", ip)
        fina_indicator = {}
        objs_delist = {}
        for i in db["stock_info"].find({}):
            obj = i["obj"]
            objs_delist[obj] = i
        c = end_date_calcor()
        del_factorlis = []
        obj_enddates = {}
        for i in db["stock_fina_indicator"].find({'update_flag':'1'}):
            if i['ann_date'] and i['end_date']:
                c.ondata(i)
                obj = i["obj"]
                if obj not in fina_indicator.keys():
                    fina_indicator[obj] = {}
                if i['end_date'] not in fina_indicator[obj].keys():
                    fina_indicator[obj][i['end_date']] = {}
                if i['ann_date'] not in fina_indicator[obj][i['end_date']].keys():
                    fina_indicator[obj][i['end_date']][i['ann_date']] = i
                else:
                    # 处理重复数据的问题
                    print(obj,i['end_date'],'存在重复数据')
                    temp =fina_indicator[obj][i['end_date']][i['ann_date']].copy()
                    for j in i.keys():
                        if j not in ['_id',"uptime","updatetimekey"]:
                            if i[j] != fina_indicator[obj][i['end_date']][i['ann_date']][j]:
                                # 如果两个都是nan
                                if i[j] != i[j] and fina_indicator[obj][i['end_date']][i['ann_date']][j] != fina_indicator[obj][i['end_date']][i['ann_date']][j]:
                                    continue
                                else:
                                    # 如果原来的值为nan,当前这个值不是nan
                                    if fina_indicator[obj][i['end_date']][i['ann_date']][j] != fina_indicator[obj][i['end_date']][i['ann_date']][j]:
                                        if i[j] == i[j]:
                                            # print(obj, i['end_date'], i['ann_date'], j, i[j],fina_indicator[obj][i['end_date']][i['ann_date']][j])
                                            fina_indicator[obj][i['end_date']][i['ann_date']][j] = i[j]
                                    # 如果原来的值不为nan，当前这个值为nan则不用管
                                    # 如果两个值都不为nan，但又不相同
                                    elif fina_indicator[obj][i['end_date']][i['ann_date']][j] == fina_indicator[obj][i['end_date']][i['ann_date']][j]:
                                        if i[j] == i[j]:
                                            #print(obj,i['end_date'],i['ann_date'],j,i[j],fina_indicator[obj][i['end_date']][i['ann_date']][j])
                                            count1 = 0
                                            for temp_i in temp.keys():
                                                if temp[temp_i] != temp[temp_i]:
                                                    count1 = count1 + 1
                                            count2 = 0
                                            for temp_j in i.keys():
                                                if i[temp_j] != i[temp_j]:
                                                    count2 =count2 + 1
                                            if count1  > count2 :
                                                fina_indicator[obj][i['end_date']][i['ann_date']] = i
                                                break
                                            elif count1 < count2:
                                                fina_indicator[obj][i['end_date']][i['ann_date']] = temp
                                                break
                                            elif count1 == count2:
                                                print('data wrong ',obj,i['end_date'],i['ann_date'],j)
                                                if j not in del_factorlis:
                                                    del_factorlis.append(j)
                                                if obj not in obj_enddates.keys():
                                                    obj_enddates[obj] = {}
                                                if i['end_date'] not in obj_enddates[obj].keys():
                                                    obj_enddates[obj][i['end_date']] = []
                                                if j not in obj_enddates[obj][i['end_date']]:
                                                    obj_enddates[obj][i['end_date']].append(j)
        c.calc_lastend_date()
        return fina_indicator,c,objs_delist
    def calc_data(self,name,fina_indicator,c,objs_delist,location,start = None,end=None,issave=False,region="C"):

        if end   is None:
            end1 = '2022-01-15'
        else:
            end1 = end
        if start:
            start1 = start
        else:
            start1 = '2015-01-01'

        rst = {}
        for d in get_calender(start1, end1,region=region):
            d_str = str(d)[:10]
            d_strs = d_str[:4] + d_str[5:7] + d_str[8:10]
            if d not in rst.keys():
                rst[d]={}
            for obj,fina_v in fina_indicator.items():
                if obj not in objs_delist.keys():
                    continue
                ds = objs_delist[obj]['list_date']
                de = objs_delist[obj]['delist_date']
                if de   is None:
                    de = '21000101'
                if ds > d_strs or de < d_strs:
                    continue
                end_date1 = c.get_lastend_date(obj, d_strs)
                if end_date1:
                    fina_v = fina_v[end_date1]
                    dates = list(fina_v.keys())
                    dates.sort()
                    fina_v = fina_v[dates[-1]]
                    if obj not in rst[d].keys():
                        value = fina_v[name[5:]]
                        if value   is not None:
                            rst[d][obj] = value
        if issave:
            print('save '+name)
            save(rst,os.path.join(location,name))
            print('save finish!')
        return rst


class index_fina_indicator:
    def update(self,ip,factor_url,k1d_url,freq="%Y%m",start_date=None,region="C"):
        from jili.data.db import tradecal as tcl
        print("update", 'index_fina_indicator')
        db = getdb_client("stock", ip)
        factor_list = ['index_PE_TTM','index_PB_LF','index_PS_TTM']
        fina_indicator = self.load_indexfina_indicator(ip)
        for name in tqdm(factor_list):
            kbase_url = os.path.join(k1d_url, "hfq", "k1d")
            base_url = os.path.join(factor_url, "bydate", name)
            base_url1 = os.path.join(factor_url, name + ".pkl")
            isadd = True
            url_bymonth = os.path.join(kbase_url, "bymonth")
            l = os.listdir(url_bymonth)
            l.sort()
            data = load(os.path.join(url_bymonth, l[-1]))
            maxdate_k = max(list(data.keys()))
            if not os.path.exists(base_url):
                if not os.path.exists(base_url1):
                    isadd = False
                    print("因子不存在,初始计算", name)
                    self.calc_data(name,fina_indicator,factor_url,start=None,end=None,issave=True,region=region)
                else:
                    print("因子存在,按日期清洗", freq, name)
                factort = factordata("test")
                factort.deal2bydate(factor_url, [name])
            if isadd:
                if start_date:
                    maxdate = str2datetime(start_date)
                else:
                    l = os.listdir(base_url)
                    l.sort()
                    data = load(os.path.join(base_url, l[-1]))
                    maxdate = max(list(data.keys()))
                if maxdate == maxdate_k:
                    print(name, "k线与因子已经是最新", maxdate)
                else:
                    print("增量计算", maxdate, maxdate_k)
                    data = self.calc_data(name,fina_indicator,factor_url,start=maxdate,end = maxdate_k ,issave=False,region=region)
                    save_new_factordata(name, data, factor_url, freq=freq)
    def load_indexfina_indicator(self,ip):
        db = getdb_client("stock", ip)
        fina_indicator = {}
        for i in db['index_daily_base_wind'].find():
            if i['date'] not in fina_indicator.keys():
                fina_indicator[i['date']] = {}
            if i['obj'] not in fina_indicator[i['date']].keys():
                fina_indicator[i['date']][i['obj']] = i
        return fina_indicator
    def calc_data(self,name,fina_indicator,location,start = None,end=None,issave=False,region="C"):
        if end   is None:
            end1 = '2022-03-08'
        else:
            end1 = end
        if start:
            start1 = start
        else:
            start1 = '2010-01-01'

        rst = {}
        for d in get_calender(start1, end1,region=region):
            d_str = str(d)[:10]
            d_strs = d_str[:4] + d_str[5:7] + d_str[8:10]
            if d not in rst.keys():
                rst[d]={}
            if d_str in fina_indicator.keys():
                for obj,fina_v in fina_indicator[d_str].items():
                    if obj not in rst[d].keys():
                        value = fina_v[name[6:]]
                        if value   is not None:
                            rst[d][obj] = value
        if issave:
            print('save '+name)
            save(rst,os.path.join(location,name))
            print('save finish!')
        return rst

class capranking_byindustry:
    # 按按行业内 成分股（流通市值+总市值)/2排序  取最大的
    def update(self,name,factor_url,k1d_url,cap,industry,start_date=None,freq="%Y%m",region="C"):
        from jili.data.db import tradecal as tcl
        print("update",name)
        kbase_url = os.path.join(k1d_url, "hfq", "k1d")
        base_url = os.path.join(factor_url, "bydate", name)
        base_url1 = os.path.join(factor_url, name + ".pkl")
        isadd = True
        url_bymonth = os.path.join(kbase_url, "bymonth")
        l = os.listdir(url_bymonth)
        l.sort()
        data = load(os.path.join(url_bymonth, l[-1]))
        maxdate_k = max(list(data.keys()))
        if not os.path.exists(base_url):
            if not os.path.exists(base_url1):
                isadd = False
                print("因子不存在,初始计算", name)
                data=self.deal_his(name,factor_url,cap,industry,start=None,issave=True,region=region)
                save_new_factordata(name, data, factor_url, freq=freq)
            else:
                print("因子存在,按日期清洗", freq, name)
            factort = factordata("test")
            factort.deal2bydate(factor_url, [name])
        if isadd:
            if start_date:
                maxdate = str2datetime(start_date)
            else:
                l = os.listdir(base_url)
                l.sort()
                data = load(os.path.join(base_url, l[-1]))
                maxdate = max(list(data.keys()))
            if maxdate == maxdate_k:
                print(name, "k线与因子已经是最新", maxdate)
                return None
            start_date = tcl.get_diff_tradeday(maxdate, 1)
            print("增量计算", maxdate,start_date,maxdate_k)
            data=self.deal_his(name,factor_url,cap,industry,start=start_date,issave=False,region=region)
            save_new_factordata(name,data,factor_url,freq=freq)
    def deal_his(self,name,factor_url,cap1,industry1,start=None,issave=False,region="C"):
        rst = {}
        if start:
            start1 = start
        else:
            start1 = '2000-01-01'
        f1 = factordata(cap1,url=factor_url)
        f2 = factordata(industry1,url=factor_url)
        if start:
            cap = {}
            industry = {}
            for k, v in f1.iter_data(start1,):
                cap[k] = v
            for k, v in f2.iter_data(start1,):
                industry[k] = v
        else:
            cap = f1.get_all()
            industry = f2.get_all()
        calulate_dates = get_calender(start=start1,region=region)
        for d in tqdm(calulate_dates):
            if d in industry.keys() and d in cap.keys():
                if d not in rst.keys():
                    rst[d] = {}
                obj_cap = {}
                for obj in cap[d].keys():
                    if obj not in obj_cap.keys():
                        obj_cap[obj] = (cap[d][obj]['circ_mv'] + cap[d][obj]['total_mv']) / 2
                obj_cap = pd.Series(obj_cap)

                for ind,v in industry[d].items():
                    obj_temp = obj_cap.reindex(v)
                    obj_temp = pd.DataFrame(obj_temp.values, index = obj_temp.index,columns=['mv'])
                    obj_temp.dropna(inplace=True)
                    obj_temp.sort_values(['mv'], ascending=False, inplace=True)
                    obj_temp["name"] = list(range(len(obj_temp), 0, -1))
                    obj_temp["name"] = 100 * obj_temp["name"] / len(obj_temp)
                    rst[d].update(obj_temp["name"].to_dict("index"))

        # if issave:
        #     save_url=os.path.join(factor_url,name)
        #     save(rst, save_url)
        return rst

def saveindex_tomongodb(factor,factor_url,ip,db_cu):
    import jili.data.db as base_db
    keys = ['date']
    db = getdb_client("stock", ip, auth='jili:jili')
    start, have = base_db.getmongodb(db[db_cu], 'date', keys, backdays=1, datestr='%Y-%m-%d')
    print('start:',start)
    f1 = factordata(factor, url=factor_url)
    if start:
        a1 = {}
        for k, v in f1.iter_data(start):
            a1[k] = v
    else:
        a1 = f1.get_all()
    ll = []
    for d,v in tqdm(a1.items()):
        for index,vv in v.items():
            for obj in vv:
                i = {'date':str(d)[:10],'obj':obj,'index':index}
                ll.append(i)
    base_db.save_mongo_bykey(db[db_cu], ll, keys, have)
    print('finish')
def deal():
    pass
if __name__ == '__main__':
    from factor.research.factor import factordata

    ip = "127.0.0.1"
    config0.set_mongodb_ip(ip)
    factor_url = r"G:\factor\k1d"
    k1d_url = r"G:\data\stock"
    date=None
    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_wind_detail_l1", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L1", mode="0",
                               cu_level="index_calssify_wind_level",
                               cu_detail="index_calssify_wind_detail")
    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_wind_concept_detail_l1", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L1", mode="0",
                               cu_level="index_calssify_wind_concept_level",
                               cu_detail="index_calssify_wind_concept_detail")
    gen_new_wind_index1 = gen_new_wind_index1()
    gen_new_wind_index1.update("is_wind_new_theme_detail", factor_url=factor_url, k1d_url=k1d_url,
                               arg=["is_wind_detail_l1", "is_wind_concept_detail_l1"],
                               add_indexs=['884039.WI', '884045.WI'], start_date=date, freq="%Y%m")
    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_cj_detail_l1", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L1", mode="0",
                               cu_level="index_calssify_cj_level",
                               cu_detail="index_calssify_cj_detail")
    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_cj_detail_l2", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L2", mode="0",
                               cu_level="index_calssify_cj_level",
                               cu_detail="index_calssify_cj_detail")
    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_cj_detail_l3", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L3", mode="0",
                               cu_level="index_calssify_cj_level",
                               cu_detail="index_calssify_cj_detail")

    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_zx_detail_l1", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L1", mode="0",
                               cu_level="index_calssify_zx_level",
                               cu_detail="index_calssify_zx_detail")
    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_zx_detail_l2", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L2", mode="0",
                               cu_level="index_calssify_zx_level",
                               cu_detail="index_calssify_zx_detail")
    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_zx_detail_l3", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L3", mode="0",
                               cu_level="index_calssify_zx_level",
                               cu_detail="index_calssify_zx_detail")

    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_sw_detail_l1", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L1", mode="0",
                               cu_level="index_calssify_sw_level_new",
                               cu_detail="index_calssify_sw_detail_new")
    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_sw_detail_l2", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L2", mode="0",
                               cu_level="index_calssify_sw_level_new",
                               cu_detail="index_calssify_sw_detail_new")
    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_sw_detail_l3", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L3", mode="0",
                               cu_level="index_calssify_sw_level_new",
                               cu_detail="index_calssify_sw_detail_new")

    index_detail_byday1 = index_detail_byday()
    index_detail_byday1.update("is_base_index_detail", factor_url, k1d_url=k1d_url,
                               ip=ip, start_date=date, freq="%Y%m", end_date=None, level="L1", mode="0",
                               cu_level="index_calssify_base_level",
                               cu_detail="index_calssify_base_detail")
#    trade_days1=trade_days()
#    trade_days1.update(ip="192.168.10.44",factor_url=r"D:\k1d", k1d_url=r"G:\data\stock",start_date=None,freq="%Y%m")

#    stock_base_info1=stock_base_info()
#    stock_base_info1.update(ip="127.0.0.1",factor_url=r"G:\factor\test", k1d_url=r"G:\data\stock",freq="%Y%m",start_date=None)
#     f = factordata("trade_days")
#     f.check_data(r"G:\factor\k1d", r"D:\k1d", "20210701", "20211015")

    # new_high1=new_high()
    # new_high1.update("new_high_days",r"G:\factor\test", k1d_url=r"G:\data\stock",args={"price":"close"},freq="%Y%m",start_date=None)
    # f = factordata("new_high_days")
    # f.check_data(r"G:\factor\test", r"G:\factor\k1d\new1", "20210701", "20210731")
    # new_high1.update("new_high_days_byhigh", r"G:\factor\test", k1d_url=r"G:\data\stock", args={"price":"high"}, freq="%Y%m",
    #                  start_date=None)

    # index_detail_byday1=index_detail_byday()
    # index_detail_byday1.update("is_wind_concept_detail_l1", r"G:\factor\test", k1d_url=r"G:\data\stock",
    #                                  ip="127.0.0.1", start_date=None, freq="%Y%m", end_date=None, level="L1", mode="0",
    #                                  cu_level="index_calssify_wind_concept_level",
    #                                  cu_detail="index_calssify_wind_concept_detail")
    # index_detail_byday1.update("is_wind_detail_l1", r"G:\factor\test", k1d_url=r"G:\data\stock",
    #                                  ip="127.0.0.1", start_date=None, freq="%Y%m", end_date=None, level="L1", mode="0",
    #                                  cu_level="index_calssify_wind_level",
    #                                  cu_detail="index_calssify_wind_detail")


    # trade_days1=trade_days()
    # trade_days1.update(r"G:\factor\test",k1d_url=r"G:\data\stock",ip="127.0.0.1",start_date=None,freq="%Y%m")


    # is_st1=is_st()
    # is_st1.update(r"G:\factor\test",k1d_url=r"G:\data\stock",ip="127.0.0.1",start_date="20060101",freq="%Y%m")